package com.crawler.pageProcessor;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

import java.util.Collections;
import java.util.List;

public class DouBanReadPageProcessor implements PageProcessor {

    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);

    @Override
    public void process(Page page) {
        // 解析页面，提取数据
        List<String> bookNames = page.getHtml().xpath("//*[@id='subject_list']//h2/a/text()").all();
        List<String> bookInfos = page.getHtml()
                .xpath("//*[@id='subject_list']//div[@class='pub']/text()").all();

        // 将数据打印到控制台
        for (int i = 0; i < bookNames.size(); i++) {
            System.out.println("书名：" + bookNames.get(i) + " 信息：" + bookInfos.get(i));
        }

        // 添加下一页的URL，继续爬取
        List<String> links = page.getHtml().links()
                .regex("https://book.douban.com/tag/小说\\?start=\\d+&type=T").all();
        if(links.size() > 0) {
            System.out.println("**************************************");
            // 获取最后一个元素
            String nextPageLink = links.get(links.size() - 1);
            page.addTargetRequests(Collections.singletonList(nextPageLink));
        }
    }

    @Override
    public Site getSite() {
        return site;
    }

}
